{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Concatenating multiple feature extraction methods"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adapted from http://scikit-learn.org/stable/auto_examples/feature_stacker.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In many real-world examples, there are many ways to extract features from a dataset. Often it is beneficial to combine several methods to obtain good performance. This example shows how to use FeatureUnion to combine features obtained by PCA and univariate selection.\n",
    "\n",
    "Combining features using this transformer has the benefit that it allows cross validation and grid searches over the whole process.\n",
    "\n",
    "The combination used in this example is not particularly helpful on this dataset and is only used to illustrate the usage of FeatureUnion."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "using ScikitLearn\n",
    "using ScikitLearn.GridSearch: GridSearchCV\n",
    "using ScikitLearn.Pipelines: Pipeline, FeatureUnion\n",
    "\n",
    "@sk_import svm: SVC\n",
    "@sk_import datasets: load_iris\n",
    "@sk_import decomposition: PCA\n",
    "@sk_import feature_selection: SelectKBest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 18 candidates, totalling 54 fits\n",
      "[CV] features__pca__n_components=1, svm__C=0.1, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=1, svm__C=0.1, features__univ_select__k=1, score=0.96078  -  0.1s\n",
      "[CV] features__pca__n_components=1, svm__C=0.1, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=1, svm__C=0.1, features__univ_select__k=1, score=0.90196  -  0.0s\n",
      "[CV] features__pca__n_components=1, svm__C=0.1, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=1, svm__C=0.1, features__univ_select__k=1, score=0.97917  -  0.0s\n",
      "[CV] features__pca__n_components=1, svm__C=1.0, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=1, svm__C=1.0, features__univ_select__k=1, score=0.94118  -  0.0s\n",
      "[CV] features__pca__n_components=1, svm__C=1.0, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=1, svm__C=1.0, features__univ_select__k=1, score=0.92157  -  0.0s\n",
      "[CV] features__pca__n_components=1, svm__C=1.0, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=1, svm__C=1.0, features__univ_select__k=1, score=0.97917  -  0.0s\n",
      "[CV] features__pca__n_components=1, svm__C=10.0, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=1, svm__C=10.0, features__univ_select__k=1, score=0.96078  -  0.0s\n",
      "[CV] features__pca__n_components=1, svm__C=10.0, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=1, svm__C=10.0, features__univ_select__k=1, score=0.92157  -  0.0s\n",
      "[CV] features__pca__n_components=1, svm__C=10.0, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=1, svm__C=10.0, features__univ_select__k=1, score=0.97917  -  0.0s\n",
      "[CV] features__pca__n_components=1, svm__C=0.1, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=1, svm__C=0.1, features__univ_select__k=2, score=0.96078  -  0.0s\n",
      "[CV] features__pca__n_components=1, svm__C=0.1, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=1, svm__C=0.1, features__univ_select__k=2, score=0.92157  -  0.0s\n",
      "[CV] features__pca__n_components=1, svm__C=0.1, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=1, svm__C=0.1, features__univ_select__k=2, score=0.97917  -  0.0s\n",
      "[CV] features__pca__n_components=1, svm__C=1.0, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=1, svm__C=1.0, features__univ_select__k=2, score=0.96078  -  0.0s\n",
      "[CV] features__pca__n_components=1, svm__C=1.0, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=1, svm__C=1.0, features__univ_select__k=2, score=0.92157  -  0.0s\n",
      "[CV] features__pca__n_components=1, svm__C=1.0, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=1, svm__C=1.0, features__univ_select__k=2, score=1.00000  -  0.0s\n",
      "[CV] features__pca__n_components=1, svm__C=10.0, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=1, svm__C=10.0, features__univ_select__k=2, score=0.98039  -  0.0s\n",
      "[CV] features__pca__n_components=1, svm__C=10.0, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=1, svm__C=10.0, features__univ_select__k=2, score=0.90196  -  0.0s\n",
      "[CV] features__pca__n_components=1, svm__C=10.0, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=1, svm__C=10.0, features__univ_select__k=2, score=1.00000  -  0.0s\n",
      "[CV] features__pca__n_components=2, svm__C=0.1, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=2, svm__C=0.1, features__univ_select__k=1, score=0.96078  -  0.0s\n",
      "[CV] features__pca__n_components=2, svm__C=0.1, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=2, svm__C=0.1, features__univ_select__k=1, score=0.90196  -  0.0s\n",
      "[CV] features__pca__n_components=2, svm__C=0.1, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=2, svm__C=0.1, features__univ_select__k=1, score=0.97917  -  0.0s\n",
      "[CV] features__pca__n_components=2, svm__C=1.0, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=2, svm__C=1.0, features__univ_select__k=1, score=0.98039  -  0.0s\n",
      "[CV] features__pca__n_components=2, svm__C=1.0, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=2, svm__C=1.0, features__univ_select__k=1, score=0.94118  -  0.0s\n",
      "[CV] features__pca__n_components=2, svm__C=1.0, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=2, svm__C=1.0, features__univ_select__k=1, score=0.97917  -  0.0s\n",
      "[CV] features__pca__n_components=2, svm__C=10.0, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=2, svm__C=10.0, features__univ_select__k=1, score=0.98039  -  0.0s\n",
      "[CV] features__pca__n_components=2, svm__C=10.0, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=2, svm__C=10.0, features__univ_select__k=1, score=0.94118  -  0.0s\n",
      "[CV] features__pca__n_components=2, svm__C=10.0, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=2, svm__C=10.0, features__univ_select__k=1, score=0.97917  -  0.0s\n",
      "[CV] features__pca__n_components=2, svm__C=0.1, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=2, svm__C=0.1, features__univ_select__k=2, score=0.98039  -  0.0s\n",
      "[CV] features__pca__n_components=2, svm__C=0.1, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=2, svm__C=0.1, features__univ_select__k=2, score=0.94118  -  0.0s\n",
      "[CV] features__pca__n_components=2, svm__C=0.1, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=2, svm__C=0.1, features__univ_select__k=2, score=0.97917  -  0.0s\n",
      "[CV] features__pca__n_components=2, svm__C=1.0, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=2, svm__C=1.0, features__univ_select__k=2, score=1.00000  -  0.0s\n",
      "[CV] features__pca__n_components=2, svm__C=1.0, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=2, svm__C=1.0, features__univ_select__k=2, score=0.96078  -  0.0s\n",
      "[CV] features__pca__n_components=2, svm__C=1.0, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=2, svm__C=1.0, features__univ_select__k=2, score=0.97917  -  0.0s\n",
      "[CV] features__pca__n_components=2, svm__C=10.0, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=2, svm__C=10.0, features__univ_select__k=2, score=0.98039  -  0.0s\n",
      "[CV] features__pca__n_components=2, svm__C=10.0, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=2, svm__C=10.0, features__univ_select__k=2, score=0.92157  -  0.0s\n",
      "[CV] features__pca__n_components=2, svm__C=10.0, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=2, svm__C=10.0, features__univ_select__k=2, score=1.00000  -  0.0s\n",
      "[CV] features__pca__n_components=3, svm__C=0.1, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=3, svm__C=0.1, features__univ_select__k=1, score=0.98039  -  0.0s\n",
      "[CV] features__pca__n_components=3, svm__C=0.1, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=3, svm__C=0.1, features__univ_select__k=1, score=0.94118  -  0.0s\n",
      "[CV] features__pca__n_components=3, svm__C=0.1, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=3, svm__C=0.1, features__univ_select__k=1, score=0.97917  -  0.0s\n",
      "[CV] features__pca__n_components=3, svm__C=1.0, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=3, svm__C=1.0, features__univ_select__k=1, score=1.00000  -  0.0s\n",
      "[CV] features__pca__n_components=3, svm__C=1.0, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=3, svm__C=1.0, features__univ_select__k=1, score=0.94118  -  0.0s\n",
      "[CV] features__pca__n_components=3, svm__C=1.0, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=3, svm__C=1.0, features__univ_select__k=1, score=0.97917  -  0.0s\n",
      "[CV] features__pca__n_components=3, svm__C=10.0, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=3, svm__C=10.0, features__univ_select__k=1, score=1.00000  -  0.0s\n",
      "[CV] features__pca__n_components=3, svm__C=10.0, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=3, svm__C=10.0, features__univ_select__k=1, score=0.92157  -  0.0s\n",
      "[CV] features__pca__n_components=3, svm__C=10.0, features__univ_select__k=1\n",
      "[CV] features__pca__n_components=3, svm__C=10.0, features__univ_select__k=1, score=1.00000  -  0.0s\n",
      "[CV] features__pca__n_components=3, svm__C=0.1, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=3, svm__C=0.1, features__univ_select__k=2, score=0.98039  -  0.0s\n",
      "[CV] features__pca__n_components=3, svm__C=0.1, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=3, svm__C=0.1, features__univ_select__k=2, score=0.94118  -  0.0s\n",
      "[CV] features__pca__n_components=3, svm__C=0.1, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=3, svm__C=0.1, features__univ_select__k=2, score=0.97917  -  0.0s\n",
      "[CV] features__pca__n_components=3, svm__C=1.0, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=3, svm__C=1.0, features__univ_select__k=2, score=1.00000  -  0.0s\n",
      "[CV] features__pca__n_components=3, svm__C=1.0, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=3, svm__C=1.0, features__univ_select__k=2, score=0.96078  -  0.0s\n",
      "[CV] features__pca__n_components=3, svm__C=1.0, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=3, svm__C=1.0, features__univ_select__k=2, score=0.97917  -  0.0s\n",
      "[CV] features__pca__n_components=3, svm__C=10.0, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=3, svm__C=10.0, features__univ_select__k=2, score=1.00000  -  0.0s\n",
      "[CV] features__pca__n_components=3, svm__C=10.0, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=3, svm__C=10.0, features__univ_select__k=2, score=0.92157  -  0.0s\n",
      "[CV] features__pca__n_components=3, svm__C=10.0, features__univ_select__k=2\n",
      "[CV] features__pca__n_components=3, svm__C=10.0, features__univ_select__k=2, score=1.00000  -  0.0s\n",
      "ScikitLearn.Skcore.Pipeline(Tuple{Any,Any}[(\"features\",ScikitLearn.Skcore.FeatureUnion(Tuple{Any,Any}[(\"pca\",PyObject PCA(copy=True, n_components=2, whiten=False)),(\"univ_select\",PyObject SelectKBest(k=2, score_func=<function f_classif at 0x31f44db90>))],1,nothing)),(\"svm\",PyObject SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape=None, degree=3, gamma='auto', kernel='linear',\n",
      "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False))])"
     ]
    }
   ],
   "source": [
    "# Original Python Author: Andreas Mueller <amueller@ais.uni-bonn.de>\n",
    "#\n",
    "# License: BSD 3 clause\n",
    "\n",
    "iris = load_iris()\n",
    "\n",
    "X, y = iris[\"data\"], iris[\"target\"]\n",
    "\n",
    "# This dataset is way to high-dimensional. Better do PCA:\n",
    "pca = PCA(n_components=2)\n",
    "\n",
    "# Maybe some original features where good, too?\n",
    "selection = SelectKBest(k=1)\n",
    "\n",
    "# Build estimator from PCA and Univariate selection:\n",
    "\n",
    "combined_features = FeatureUnion([(\"pca\", pca), (\"univ_select\", selection)])\n",
    "\n",
    "# Use combined features to transform dataset:\n",
    "X_features = transform(fit!(combined_features, X, y), X)\n",
    "\n",
    "svm = SVC(kernel=\"linear\")\n",
    "\n",
    "# Do grid search over k, n_components and C:\n",
    "\n",
    "pipeline = Pipeline([(\"features\", combined_features), (\"svm\", svm)])\n",
    "\n",
    "param_grid = Dict(:features__pca__n_components=>[1, 2, 3],\n",
    "                  :features__univ_select__k=>[1, 2],\n",
    "                  :svm__C=>[0.1, 1, 10])\n",
    "\n",
    "grid_search = GridSearchCV(pipeline, param_grid; verbose=10, refit=true)\n",
    "\n",
    "fit!(grid_search, X, y)\n",
    "\n",
    "print(grid_search.best_estimator_)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 0.4.5",
   "language": "julia",
   "name": "julia-0.4"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "0.4.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
